{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "词嵌入.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/yejianfeng2014/keras-1/blob/master/%E8%AF%8D%E5%B5%8C%E5%85%A5.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "metadata": {
        "id": "6u6-A2Y60WGr",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 503
        },
        "outputId": "e4665a54-ad7a-42ab-a5a7-ecc4c5b7744c"
      },
      "cell_type": "code",
      "source": [
        "from keras.preprocessing.text import one_hot\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "from keras.models import Sequential\n",
        "from keras.layers import Dense\n",
        "from keras.layers import Flatten\n",
        "from keras.layers.embeddings import Embedding\n",
        "# define documents\n",
        "docs = ['Well done!',\n",
        "\t\t'Good work',\n",
        "\t\t'Great effort',\n",
        "\t\t'nice work',\n",
        "\t\t'Excellent!',\n",
        "\t\t'Weak',\n",
        "\t\t'Poor effort!',\n",
        "\t\t'not good',\n",
        "\t\t'poor work',\n",
        "\t\t'Could have done better.']\n",
        "# define class labels\n",
        "labels = [1,1,1,1,1,0,0,0,0,0]\n",
        "# integer encode the documents\n",
        "vocab_size = 50\n",
        "encoded_docs = [one_hot(d, vocab_size) for d in docs]\n",
        "print(encoded_docs)\n",
        "# pad documents to a max length of 4 words\n",
        "max_length = 4\n",
        "padded_docs = pad_sequences(encoded_docs, maxlen=max_length, padding='post')\n",
        "print(padded_docs)\n",
        "# define the model\n",
        "model = Sequential()\n",
        "model.add(Embedding(vocab_size, 8, input_length=max_length))\n",
        "model.add(Flatten())\n",
        "model.add(Dense(1, activation='sigmoid'))\n",
        "# compile the model\n",
        "model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['acc'])\n",
        "# summarize the model\n",
        "print(model.summary())\n",
        "# fit the model\n",
        "model.fit(padded_docs, labels, epochs=50, verbose=0)\n",
        "# evaluate the model\n",
        "loss, accuracy = model.evaluate(padded_docs, labels, verbose=0)\n",
        "print('Accuracy: %f' % (accuracy*100))\n"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "[[47, 28], [1, 15], [19, 21], [24, 15], [43], [44], [40, 21], [3, 1], [40, 15], [28, 15, 28, 28]]\n",
            "[[47 28  0  0]\n",
            " [ 1 15  0  0]\n",
            " [19 21  0  0]\n",
            " [24 15  0  0]\n",
            " [43  0  0  0]\n",
            " [44  0  0  0]\n",
            " [40 21  0  0]\n",
            " [ 3  1  0  0]\n",
            " [40 15  0  0]\n",
            " [28 15 28 28]]\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "embedding_1 (Embedding)      (None, 4, 8)              400       \n",
            "_________________________________________________________________\n",
            "flatten_1 (Flatten)          (None, 32)                0         \n",
            "_________________________________________________________________\n",
            "dense_1 (Dense)              (None, 1)                 33        \n",
            "=================================================================\n",
            "Total params: 433\n",
            "Trainable params: 433\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "None\n",
            "Accuracy: 100.000000\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "OOS_abAm-Wj8",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 263
        },
        "outputId": "20cc8867-cbe9-49db-c6ea-02675e681377"
      },
      "cell_type": "code",
      "source": [
        "'''This script loads pre-trained word embeddings (GloVe embeddings)\n",
        "into a frozen Keras Embedding layer, and uses it to\n",
        "train a text classification model on the 20 Newsgroup dataset\n",
        "(classification of newsgroup messages into 20 different categories).\n",
        "GloVe embedding data can be found at:\n",
        "http://nlp.stanford.edu/data/glove.6B.zip\n",
        "(source page: http://nlp.stanford.edu/projects/glove/)\n",
        "20 Newsgroup data can be found at:\n",
        "http://www.cs.cmu.edu/afs/cs.cmu.edu/project/theo-20/www/data/news20.html\n",
        "'''\n",
        "\n",
        "from __future__ import print_function\n",
        "\n",
        "import os\n",
        "import sys\n",
        "import numpy as np\n",
        "from keras.preprocessing.text import Tokenizer\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "from keras.utils import to_categorical\n",
        "from keras.layers import Dense, Input, GlobalMaxPooling1D\n",
        "from keras.layers import Conv1D, MaxPooling1D, Embedding\n",
        "from keras.models import Model\n",
        "from keras.initializers import Constant\n",
        "\n",
        "\n",
        "BASE_DIR = ''\n",
        "GLOVE_DIR = os.path.join(BASE_DIR, 'glove.6B')\n",
        "TEXT_DATA_DIR = os.path.join(BASE_DIR, '20_newsgroup')\n",
        "MAX_SEQUENCE_LENGTH = 1000\n",
        "MAX_NUM_WORDS = 20000\n",
        "EMBEDDING_DIM = 100\n",
        "VALIDATION_SPLIT = 0.2\n",
        "\n",
        "# first, build index mapping words in the embeddings set\n",
        "# to their embedding vector\n",
        "\n",
        "print('Indexing word vectors.')\n",
        "\n",
        "embeddings_index = {}\n",
        "with open(os.path.join(GLOVE_DIR, 'glove.6B.100d.txt')) as f:\n",
        "    for line in f:\n",
        "        values = line.split()\n",
        "        word = values[0]\n",
        "        coefs = np.asarray(values[1:], dtype='float32')\n",
        "        embeddings_index[word] = coefs\n",
        "\n",
        "print('Found %s word vectors.' % len(embeddings_index))\n",
        "\n",
        "# second, prepare text samples and their labels\n",
        "print('Processing text dataset')\n",
        "\n",
        "texts = []  # list of text samples\n",
        "labels_index = {}  # dictionary mapping label name to numeric id\n",
        "labels = []  # list of label ids\n",
        "for name in sorted(os.listdir(TEXT_DATA_DIR)):\n",
        "    path = os.path.join(TEXT_DATA_DIR, name)\n",
        "    if os.path.isdir(path):\n",
        "        label_id = len(labels_index)\n",
        "        labels_index[name] = label_id\n",
        "        for fname in sorted(os.listdir(path)):\n",
        "            if fname.isdigit():\n",
        "                fpath = os.path.join(path, fname)\n",
        "                args = {} if sys.version_info < (3,) else {'encoding': 'latin-1'}\n",
        "                with open(fpath, **args) as f:\n",
        "                    t = f.read()\n",
        "                    i = t.find('\\n\\n')  # skip header\n",
        "                    if 0 < i:\n",
        "                        t = t[i:]\n",
        "                    texts.append(t)\n",
        "                labels.append(label_id)\n",
        "\n",
        "print('Found %s texts.' % len(texts))\n",
        "\n",
        "# finally, vectorize the text samples into a 2D integer tensor\n",
        "tokenizer = Tokenizer(num_words=MAX_NUM_WORDS)\n",
        "tokenizer.fit_on_texts(texts)\n",
        "sequences = tokenizer.texts_to_sequences(texts)\n",
        "\n",
        "word_index = tokenizer.word_index\n",
        "print('Found %s unique tokens.' % len(word_index))\n",
        "\n",
        "data = pad_sequences(sequences, maxlen=MAX_SEQUENCE_LENGTH)\n",
        "\n",
        "labels = to_categorical(np.asarray(labels))\n",
        "print('Shape of data tensor:', data.shape)\n",
        "print('Shape of label tensor:', labels.shape)\n",
        "\n",
        "# split the data into a training set and a validation set\n",
        "indices = np.arange(data.shape[0])\n",
        "np.random.shuffle(indices)\n",
        "data = data[indices]\n",
        "labels = labels[indices]\n",
        "num_validation_samples = int(VALIDATION_SPLIT * data.shape[0])\n",
        "\n",
        "x_train = data[:-num_validation_samples]\n",
        "y_train = labels[:-num_validation_samples]\n",
        "x_val = data[-num_validation_samples:]\n",
        "y_val = labels[-num_validation_samples:]\n",
        "\n",
        "print('Preparing embedding matrix.')\n",
        "\n",
        "# prepare embedding matrix\n",
        "num_words = min(MAX_NUM_WORDS, len(word_index)) + 1\n",
        "embedding_matrix = np.zeros((num_words, EMBEDDING_DIM))\n",
        "for word, i in word_index.items():\n",
        "    if i > MAX_NUM_WORDS:\n",
        "        continue\n",
        "    embedding_vector = embeddings_index.get(word)\n",
        "    if embedding_vector is not None:\n",
        "        # words not found in embedding index will be all-zeros.\n",
        "        embedding_matrix[i] = embedding_vector\n",
        "\n",
        "# load pre-trained word embeddings into an Embedding layer\n",
        "# note that we set trainable = False so as to keep the embeddings fixed\n",
        "embedding_layer = Embedding(num_words,\n",
        "                            EMBEDDING_DIM,\n",
        "                            embeddings_initializer=Constant(embedding_matrix),\n",
        "                            input_length=MAX_SEQUENCE_LENGTH,\n",
        "                            trainable=False)\n",
        "\n",
        "print('Training model.')\n",
        "\n",
        "# train a 1D convnet with global maxpooling\n",
        "sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')\n",
        "embedded_sequences = embedding_layer(sequence_input)\n",
        "x = Conv1D(128, 5, activation='relu')(embedded_sequences)\n",
        "x = MaxPooling1D(5)(x)\n",
        "x = Conv1D(128, 5, activation='relu')(x)\n",
        "x = MaxPooling1D(5)(x)\n",
        "x = Conv1D(128, 5, activation='relu')(x)\n",
        "x = GlobalMaxPooling1D()(x)\n",
        "x = Dense(128, activation='relu')(x)\n",
        "preds = Dense(len(labels_index), activation='softmax')(x)\n",
        "\n",
        "model = Model(sequence_input, preds)\n",
        "model.compile(loss='categorical_crossentropy',\n",
        "              optimizer='rmsprop',\n",
        "              metrics=['acc'])\n",
        "\n",
        "model.fit(x_train, y_train,\n",
        "          batch_size=128,\n",
        "          epochs=10,\n",
        "          validation_data=(x_val, y_val))"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Indexing word vectors.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "FileNotFoundError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-3-110e96a30042>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     38\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     39\u001b[0m \u001b[0membeddings_index\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 40\u001b[0;31m \u001b[0;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mGLOVE_DIR\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'glove.6B.100d.txt'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     41\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mline\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     42\u001b[0m         \u001b[0mvalues\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mline\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'glove.6B/glove.6B.100d.txt'"
          ]
        }
      ]
    }
  ]
}